{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pytorch_data_loader\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "backup.pkl  evidence.pkl  train.pkl\r\n"
     ]
    }
   ],
   "source": [
    "!ls data/small"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "claims_dict = joblib.load(\"new_claims_dict.pkl\")\n",
    "train = joblib.load(\"data/large/train.pkl\")\n",
    "evidences = joblib.load(\"data/small/evidence.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Distinct Claims 109810\n",
      "Num Data Points 125050\n"
     ]
    }
   ],
   "source": [
    "dataset = pytorch_data_loader.WikiDataset(train, claims_dict, sparse_evidences=evidences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting line_profiler\n",
      "  Using cached https://files.pythonhosted.org/packages/14/fc/ecf4e238bb601ff829068e5a72cd1bd67b0ee0ae379db172eb6a0779c6b6/line_profiler-2.1.2.tar.gz\n",
      "Requirement already satisfied: IPython>=0.13 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from line_profiler) (7.1.1)\n",
      "Requirement already satisfied: pygments in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (2.2.0)\n",
      "Requirement already satisfied: traitlets>=4.2 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (4.3.2)\n",
      "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (2.0.7)\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (4.6.0)\n",
      "Requirement already satisfied: jedi>=0.10 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (0.13.1)\n",
      "Requirement already satisfied: decorator in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (4.3.0)\n",
      "Requirement already satisfied: setuptools>=18.5 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (40.6.2)\n",
      "Requirement already satisfied: pickleshare in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (0.7.5)\n",
      "Requirement already satisfied: backcall in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from IPython>=0.13->line_profiler) (0.1.0)\n",
      "Requirement already satisfied: six in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from traitlets>=4.2->IPython>=0.13->line_profiler) (1.11.0)\n",
      "Requirement already satisfied: ipython-genutils in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from traitlets>=4.2->IPython>=0.13->line_profiler) (0.2.0)\n",
      "Requirement already satisfied: wcwidth in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from prompt-toolkit<2.1.0,>=2.0.0->IPython>=0.13->line_profiler) (0.1.7)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from pexpect; sys_platform != \"win32\"->IPython>=0.13->line_profiler) (0.6.0)\n",
      "Requirement already satisfied: parso>=0.3.0 in /data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages (from jedi>=0.10->IPython>=0.13->line_profiler) (0.3.1)\n",
      "Building wheels for collected packages: line-profiler\n",
      "  Running setup.py bdist_wheel for line-profiler ... \u001b[?25lerror\n",
      "  Complete output from command /data/sls/u/urop/mnadeem/my_venv/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-install-0t0yzydc/line-profiler/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" bdist_wheel -d /tmp/pip-wheel-qo3m3u7b --python-tag cp35:\n",
      "  Could not import Cython. Using the available pre-generated C file.\n",
      "  running bdist_wheel\n",
      "  running build\n",
      "  running build_py\n",
      "  creating build\n",
      "  creating build/lib.linux-x86_64-3.5\n",
      "  copying line_profiler.py -> build/lib.linux-x86_64-3.5\n",
      "  copying kernprof.py -> build/lib.linux-x86_64-3.5\n",
      "  copying line_profiler_py35.py -> build/lib.linux-x86_64-3.5\n",
      "  running build_ext\n",
      "  building '_line_profiler' extension\n",
      "  creating build/temp.linux-x86_64-3.5\n",
      "  x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.5m -I/data/sls/u/urop/mnadeem/my_venv/include/python3.5m -c _line_profiler.c -o build/temp.linux-x86_64-3.5/_line_profiler.o\n",
      "  _line_profiler.c:4:20: fatal error: Python.h: No such file or directory\n",
      "  compilation terminated.\n",
      "  error: command 'x86_64-linux-gnu-gcc' failed with exit status 1\n",
      "  \n",
      "  ----------------------------------------\n",
      "\u001b[31m  Failed building wheel for line-profiler\u001b[0m\n",
      "\u001b[?25h  Running setup.py clean for line-profiler\n",
      "Failed to build line-profiler\n",
      "Installing collected packages: line-profiler\n",
      "  Running setup.py install for line-profiler ... \u001b[?25lerror\n",
      "    Complete output from command /data/sls/u/urop/mnadeem/my_venv/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-install-0t0yzydc/line-profiler/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" install --record /tmp/pip-record-udvz6d0l/install-record.txt --single-version-externally-managed --compile --install-headers /data/sls/u/urop/mnadeem/my_venv/include/site/python3.5/line-profiler:\n",
      "    Could not import Cython. Using the available pre-generated C file.\n",
      "    running install\n",
      "    running build\n",
      "    running build_py\n",
      "    creating build\n",
      "    creating build/lib.linux-x86_64-3.5\n",
      "    copying line_profiler.py -> build/lib.linux-x86_64-3.5\n",
      "    copying kernprof.py -> build/lib.linux-x86_64-3.5\n",
      "    copying line_profiler_py35.py -> build/lib.linux-x86_64-3.5\n",
      "    running build_ext\n",
      "    building '_line_profiler' extension\n",
      "    creating build/temp.linux-x86_64-3.5\n",
      "    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.5m -I/data/sls/u/urop/mnadeem/my_venv/include/python3.5m -c _line_profiler.c -o build/temp.linux-x86_64-3.5/_line_profiler.o\n",
      "    _line_profiler.c:4:20: fatal error: Python.h: No such file or directory\n",
      "    compilation terminated.\n",
      "    error: command 'x86_64-linux-gnu-gcc' failed with exit status 1\n",
      "    \n",
      "    ----------------------------------------\n",
      "\u001b[31mCommand \"/data/sls/u/urop/mnadeem/my_venv/bin/python3 -u -c \"import setuptools, tokenize;__file__='/tmp/pip-install-0t0yzydc/line-profiler/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\\r\\n', '\\n');f.close();exec(compile(code, __file__, 'exec'))\" install --record /tmp/pip-record-udvz6d0l/install-record.txt --single-version-externally-managed --compile --install-headers /data/sls/u/urop/mnadeem/my_venv/include/site/python3.5/line-profiler\" failed with error code 1 in /tmp/pip-install-0t0yzydc/line-profiler/\u001b[0m\n",
      "\u001b[?25h"
     ]
    }
   ],
   "source": [
    "!pip3 install line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named 'line_profiler'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-df8a33df4eaf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'load_ext'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'line_profiler'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line, _stack_depth)\u001b[0m\n\u001b[1;32m   2285\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2286\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2287\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2288\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2289\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<decorator-gen-65>\u001b[0m in \u001b[0;36mload_ext\u001b[0;34m(self, module_str)\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    185\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages/IPython/core/magics/extension.py\u001b[0m in \u001b[0;36mload_ext\u001b[0;34m(self, module_str)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodule_str\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mUsageError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Missing module name.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextension_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_extension\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule_str\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'already loaded'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/site-packages/IPython/core/extensions.py\u001b[0m in \u001b[0;36mload_extension\u001b[0;34m(self, module_str)\u001b[0m\n\u001b[1;32m     78\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmodule_str\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mprepended_to_syspath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mipython_extension_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m                     \u001b[0mmod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule_str\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mmod\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__file__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mipython_extension_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m                         print((\"Loading extensions from {dir} is deprecated. \"\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/importlib/__init__.py\u001b[0m in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m    124\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mlevel\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_bootstrap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gcd_import\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_gcd_import\u001b[0;34m(name, package, level)\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;32m/data/sls/u/urop/mnadeem/my_venv/lib/python3.5/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load_unlocked\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named 'line_profiler'"
     ]
    }
   ],
   "source": [
    "%load_ext line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
